package traverser;

/**
 * CacheLinkProcessor.java
 *
 * Extension of link processor that stores visited
 * pages in a cache for quick retrieval.
 */

import java.io.File;
import java.io.FileNotFoundException;
import java.io.FileWriter;
import java.io.IOException;
import java.io.InputStream;
import java.net.URL;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Comparator;
import java.util.HashSet;
import java.util.List;
import java.util.Random;
import java.util.Scanner;
import java.util.Set;

public class CacheLinkProcessor {
	public static final boolean USE_CACHE = true;
	public static final int  MAX_ATTEMPTS = 5;
	public static final String HOME = System.getProperty("user.home");
	public static final String cacheFolder = HOME + "/scratch/wccache/";

	private String htmlCode = ""; // HTML code of the website
	private String[] links; // List of all of the links on the given webpage

	private String website = "";
	private String filename = "";
	private boolean cached = false;



	private FrequencyCounter fq = null;

	private Random rnj;

	// Timers for various functions
	private static long HTMLtime = 0;
	private static long extractTime = 0;
	private static long processTime = 0;
	private static int misses = 0;
	private static int querries = 0;

	public CacheLinkProcessor(String website) {
		querries++;
		rnj = new Random();

		// Check if the cache directory is initialized
		File cache = new File(cacheFolder);
		if (!cache.exists()) {
			cache.mkdir();
		}

		this.website = website;

		// Record how long stuff takes
		long startTime, endTime;

		filename = website.replace('/', ' ');

		if (!USE_CACHE || !loadLinks(filename)) {
			misses++;
			startTime = System.currentTimeMillis();
			getHTML(website);
			endTime = System.currentTimeMillis();
			HTMLtime += endTime - startTime;

			startTime = System.currentTimeMillis();
			extractLinks(htmlCode);
			endTime = System.currentTimeMillis();
			extractTime += endTime - startTime;
		}
	}

	public CacheLinkProcessor(String website, FrequencyCounter fq) {
		this(website);
		this.fq = fq;
	}

	public CacheLinkProcessor(String website, int seed) {
		this(website);
		rnj = new Random(seed);
	}


	// Loads the list of liks fromt the cache.  Returns true if successful, or false if not.
	public boolean loadLinks(String filename) {
		File file = new File(cacheFolder + filename.substring(29) + ".links");
		if (!file.exists()) {
			return false;
		}

		cached = true;

		Scanner sc = null;
		try {
			sc = new Scanner(file);
		} catch (FileNotFoundException e) {
			e.printStackTrace();
			return false;
		}

		if (!sc.hasNextInt()) {
			// Bad format
			return false;
		}

		//System.out.println("Successfully loaded file " + file.toString());

		links = new String[sc.nextInt()];
		sc.nextLine();  // discard the rest of the line

		int i = 0;
		while (sc.hasNextLine()) {
			links[i] = sc.nextLine();
			i++;
		}
		return true;
	}

	// gets HTML code from website and puts it in htmlCode
	public void getHTML(String website) {
		URL site = null;
		InputStream contentStream = null;

		// Attempt to connect to the page MAX_ATTEMPTS times
		int attempts;
		for (attempts = 0; attempts < MAX_ATTEMPTS; attempts++) {
			try {
				site = new URL(website);
				contentStream = site.openStream();
				break;
			} catch (IOException e) {
				System.err.println("Error connecting to " + website + " - attempt " + attempts + ".  Message:");
				System.err.println(e.getMessage());
			}
		}

		if (attempts >= MAX_ATTEMPTS) {
			System.out.println("Could not connect to " + website + " after " + attempts + " attempts.  Abandoning...");
			// Act as though there are no links on this page
			htmlCode = "";
			return;
		}

		Scanner sc = new Scanner(contentStream);
		while (sc.hasNext()) {
			htmlCode += sc.nextLine();
		}
	}

	// Extracts the links from a piece of HTML and puts them into an array
	public void extractLinks(String htmlCode) {
		String key = "<a href=\"/wiki/"; // the term we are searching for
		int keyLen = key.length();
		String link = "";

		int index = 0; // index of the search
		int start = 0; // start index of the link
		int end = 0; // end index of the link

		// Use an array list for simple implementation
		ArrayList<String> list = new ArrayList<String>();

		// runs while index is less than string length
		while (index < htmlCode.length()) {
			// searches for key terms and gets index
			start = htmlCode.indexOf(key, index);

			// if key terms found, add them to the list and update index
			if (start > 0) {
				end = htmlCode.indexOf("\"", start + keyLen);
				link = htmlCode.substring(start + keyLen, end);
				if (inValid(link)) {
					index = end;
					continue;
				}

				list.add("http://en.wikipedia.org/wiki/" + link);
				index = end;
			}
			// if key terms not found, end loop
			else if (start == -1) {
				break;
			}
		}

		// Cast from ArrayList to array is super easy.
		this.links = new String[list.size()];
		this.links = list.toArray(this.links);
		removeDuplicates();
		writeCache();
	}




	// Checks whether a link goes to an ignored page type
	public boolean inValid (String link) {
		return (link.contains("Wikipedia:")
				|| link.contains("Special:")
				|| link.contains("Help:")
				|| link.contains("File:")
				|| link.contains("Category:")
				|| link.contains("Template:")
				|| link.contains("Portal:")
				|| link.contains("Talk:")
				|| link.contains("#"));
	}

	// Writes a list of links to disk, for the cache
	public void writeCache () {
		try {
			File linkFile = new File(cacheFolder + filename.substring(29) + ".links");
			if (! linkFile.exists() ) {
				linkFile.createNewFile();
			}
			FileWriter fileWrite = new FileWriter(linkFile);

			fileWrite.write(links.length + "\n");
			for (int counter = 0; counter < links.length; counter++) {
				fileWrite.write(links[counter] + "\n");
			}
			fileWrite.close();
		} catch (IOException e) {
			System.err.println("Could not write to file: " + e.getMessage());
		}
	}

	// Remove duplicate entries from links
	public void removeDuplicates() {
		int count = links.length;
		for (int i = 0; i < count; i++) {
			for (int j = i + 1; j < count; j++) {
				if (links[i].equals(links[j])) {
					count--; // One fewer unique link
					links[j] = links[count]; // Omit the duplicate link, draw a new one from the end
					j--; // Check the just swapped in link
				}
			}
		}
		String[] noDups = new String[count];
		for (int i = 0; i < count; i++) {
			noDups[i] = links[i];
		}
		links = noDups;
	}

	// sorts linkArray - removes duplicates and then puts into alpha order
	public void alphaSort() {
		ListSorter linkSort = new ListSorter(links);
		links = linkSort.sort();
	}

	// sorts linkArray - sorts from highest frequency to lowest frequency
//	public void freqSort() {
//		if (fq == null) {
//			System.err.println("Error: frequency tracker not found!");
//			System.exit(1);
//		}
//
//		for (int i = 0; i < count(); i++) {
//			for (int j = i + 1; j < count(); j++) {
//
//				if (fq.frequency(links[i]) < fq.frequency(links[j])) {
//					String temp = links[i];
//					links[i] = links[j];
//					links[j] = temp;
//				}
//			}
//		}
//
//	}
	public void freqSort(){
		if(fq==null){
			System.err.println("Error: frequency tracker not found!");
			System.exit(1);
		}
		Quicksort(links,1,count());
	}
	public  void Quicksort(String[] a, int left, int right) {
		int i=left;
		int j=right;
		int x=fq.frequency(a[(left+right)/2]);
		while(i<=j){
			while(fq.frequency(a[i])>x) i++;
			while(x>fq.frequency(a[j])) j--;
			if(i<=j) {
				String temp;
				temp=a[i];
				a[i]=a[j];
				a[j]=temp;
				i++;
				j--;
			}
		}
		if(left<j) Quicksort(a, left, j);
		if(i<right) Quicksort(a,i,right);
	}
	

	// Shuffle
	public void setSeed(int seed) {
		rnj = new Random(seed);
	}

	public void knuthShuffle() {
		knuthShuffle(0, links.length);
	}

	private void knuthShuffle(int start, int end) {
		for (int i = start; i < end - 1; i++) {
			int j = i + (int) (rnj.nextInt(end - i - 1));
			String temp = links[i];
			links[i] = links[j];
			links[j] = temp;
		}
	}

	// Sort the links by frequency
	public void freqOrder(FrequencyCounter fq) {
		Arrays.sort(links, new FrequencyComparator(fq));
		// TODO should probably add a shuffle within frequency classes here
		
	}

	// Comparator function, allows us to compare pages based on frequency
	class FrequencyComparator implements Comparator<String> {
		FrequencyCounter fq;

		public FrequencyComparator(FrequencyCounter fq) {
			this.fq = fq;
		}

		public int compare(String page1, String page2) {
			if (fq.contains(page1) && fq.contains(page2)) {
				// We want the page with greater frequency to occur first
				return fq.frequency(page2) - fq.frequency(page1);
			}
			// At least one page has not been seen
			return 0;
		}

	}
	
	// LinkedList of links
	public String[] getLinks() {
		return links;
	}

	// How many links are there?
	public int count() {
		return links.length;
	}

	// Did we find this page in the cache?
	public boolean wasCached() {
		return cached;
	}

	/*
	    public static void setUseCache(boolean status) {
	        USE_CACHE = status;
	    }
	 */

	// Metrics
	public static long getHTMLtime() {
		return HTMLtime;
	}

	public static long getExtractTime() {
		return extractTime;
	}

	public static long getProcessTime() {
		return processTime;
	}

	public static void printMetrics() {
		System.out.println("Time cost breakdown:");
		System.out.println("Time spent on HTML:\t" + HTMLtime);
		System.out.println("Time spent on extract:\t" + extractTime);
		System.out.println("Time spent on process:\t" + processTime);
		System.out.println();
		System.out.println("Cache querries:\t" + querries);
		System.out.println("Cache misses:\t" + misses);
		System.out.println("Cache hit rate\t" + (1.0 * querries - misses) / querries);
	}

}
